{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Phase 3 Weighted Bagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\gensim\\utils.py:1197: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n",
      "Using TensorFlow backend.\n",
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\fuzzywuzzy\\fuzz.py:35: UserWarning: Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning\n",
      "  warnings.warn('Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning')\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "\n",
    "import os\n",
    "import re\n",
    "import csv\n",
    "import codecs\n",
    "import gensim\n",
    "import itertools\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import operator\n",
    "import sys\n",
    "\n",
    "from nltk import ngrams\n",
    "from collections import Counter\n",
    "from string import punctuation\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "from iwillwin.trainer.supervised_trainer import KerasModelTrainer\n",
    "from iwillwin.data_utils.data_helpers import DataTransformer, DataLoader\n",
    "from iwillwin.config import dataset_config\n",
    "from iwillwin.data_utils.feature_engineering import FeatureCreator\n",
    "\n",
    "from fuzzywuzzy import fuzz\n",
    "from nltk.corpus import stopwords\n",
    "from tqdm import tqdm\n",
    "from scipy.stats import skew, kurtosis\n",
    "from scipy.spatial.distance import cosine, cityblock, jaccard, canberra, euclidean, minkowski, braycurtis\n",
    "from nltk import word_tokenize\n",
    "\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "\n",
    "import lightgbm as lgb\n",
    "from sklearn.model_selection import train_test_split\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "import os\n",
    "import re\n",
    "import csv\n",
    "import codecs\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import operator\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "\n",
    "########################################\n",
    "## import packages\n",
    "########################################\n",
    "import os\n",
    "import re\n",
    "import csv\n",
    "import codecs\n",
    "import numpy as np\n",
    "np.random.seed(1337)\n",
    "\n",
    "import pandas as pd\n",
    "import operator\n",
    "import sys\n",
    "\n",
    "from string import punctuation\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "from iwillwin.trainer.supervised_trainer import KerasModelTrainer\n",
    "from iwillwin.data_utils.data_helpers import DataTransformer, DataLoader\n",
    "from iwillwin.config import dataset_config\n",
    "from keras.utils import to_categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache C:\\Users\\zake7\\AppData\\Local\\Temp\\jieba.cache\n",
      "Loading model cost 0.863 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[DataHelper] Apply normalization on value-type columns\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py:475: DataConversionWarning: Data with input dtype int64 was converted to float64 by MinMaxScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Doing preprocessing...\n",
      "Transforming words to indices...\n",
      "Shape of data tensor: (320552, 50) (320552, 50)\n",
      "Shape of label tensor: (320552,)\n",
      "Preprocessed.\n",
      "Number of unique words 83265\n"
     ]
    }
   ],
   "source": [
    "NB_WORDS, MAX_SEQUENCE_LENGTH = 50000, 50\n",
    "data_transformer = DataTransformer(max_num_words=NB_WORDS, max_sequence_length=MAX_SEQUENCE_LENGTH, char_level=False,\n",
    "                                   normalization=True, features_processed=True)\n",
    "trains_nns, tests_nns, labels = data_transformer.prepare_data(dual=False)\n",
    "print(\"Number of unique words\", len(data_transformer.tokenizer.index_docs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trains_meta = trains_nns[2]\n",
    "tests_meta = tests_nns[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_df = pd.read_csv('../data/dataset/train.csv')\n",
    "test_df = pd.read_csv('../data/dataset/test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nan <class 'float'>\n",
      "nan <class 'float'>\n",
      "nan <class 'float'>\n",
      "nan <class 'float'>\n",
      "nan <class 'float'>\n",
      "nan <class 'float'>\n",
      "nan <class 'float'>\n",
      "nan <class 'float'>\n"
     ]
    }
   ],
   "source": [
    "rumor_words = ['辟谣', '谣言', '勿传', '假的']\n",
    "\n",
    "def is_rumor(text):\n",
    "    if type(text) != str:\n",
    "        print(text, type(text))\n",
    "        return 0\n",
    "    for rumor_word in rumor_words:\n",
    "        if rumor_word in text:\n",
    "            return 1\n",
    "    return 0\n",
    "\n",
    "def has_split_symbol(text):\n",
    "    if type(text) != str:\n",
    "        return 0\n",
    "    if '|' in text:\n",
    "        return 1\n",
    "    return 0\n",
    "\n",
    "for df in [train_df, test_df]:\n",
    "    df['has_|'] = df['title2_zh'].apply(has_split_symbol)\n",
    "    df['has_rumor_words'] = df['title2_zh'].apply(is_rumor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_has_rumor = train_df.has_rumor_words.values\n",
    "test_has_rumor = test_df.has_rumor_words.values\n",
    "\n",
    "trick_trains_features = np.concatenate((trains_nns[2], train_has_rumor.reshape((-1, 1))), axis=1)\n",
    "trick_tests_features = np.concatenate((tests_nns[2], test_has_rumor.reshape((-1, 1))), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "oof_file_names = sorted([f for f in listdir('../data/pseudo/oofs/') if isfile(join('../data/pseudo/oofs/', f)) and f != '.gitkeep'])\n",
    "preds_file_names = [name.replace('-Train', '') for name in oof_file_names]\n",
    "\n",
    "oofs = []\n",
    "preds = []\n",
    "for name in oof_file_names:\n",
    "    oofs.append(pd.read_csv('../data/pseudo/oofs/' + name))\n",
    "for name in preds_file_names:\n",
    "    preds.append(pd.read_csv('../data/pseudo/output/' + name))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 3Embedding-3LayersDenseCNN42-NoDrop-NoClassWeighted-NoEM-Train-L0.809633-NB5000.csv\n",
      "1 3Embedding-3LayersDenseRNN42-Drop01-NoMeta-NoClassWeighted-WithEM-Train-L0.816583-NB5000.csv\n",
      "2 3Embedding-ESIM-Drop01-NoMeta-NoClassWeighted-NoEM-Train-L0.833565-NB5000.csv\n",
      "3 WordSGNS-DenseCNN5Layers-NoMeta-3P-NoEM-NoClassWeighted-3Layers-Train-L0.838202-NB100000.csv\n",
      "4 WordSGNS-ESIM-NoMeta-3P-NoEM-NoClassWeighted-3Layers-Train-L1.104962-NB100000.csv\n",
      "5 WordTC-DenseCNN5Layers-NoMeta-3P-NoEM-NoClassWeighted-3Layers-Train-L0.8440-NB100000.csv\n",
      "6 WordTC-DenseRNN-NoMeta-3P-NoEM-NoClassWeighted-3Layers-Train-L0.854586-NB100000.csv\n",
      "7 WordTC-ESIM-NoMeta-3P-NoEM-NoClassWeighted-3Layers-Train-L0.374334-NB100000.csv\n",
      "8 WordTC-Gated4GWindows-NoMeta-3P-NoEM-NoClassWeighted-3Layers-withEM-Train-L0.836860-NB100000.csv\n"
     ]
    }
   ],
   "source": [
    "for i, name in enumerate(oof_file_names):\n",
    "    print(i, name)\n",
    "    \n",
    "trains = pd.DataFrame()\n",
    "tests = pd.DataFrame()\n",
    "\n",
    "for i in range(len(oof_file_names)):\n",
    "    for label_type in ['agreed', 'disagreed', 'unrelated']:\n",
    "        trains['oofs_{}_{}'.format(i, label_type)] = oofs[i][label_type].values\n",
    "        tests['oofs_pred{}_{}'.format(i, label_type)] = preds[i][label_type].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['3Embedding-3LayersDenseCNN42-NoDrop-NoClassWeighted-NoEM-Train-L0.809633-NB5000.csv',\n",
       " '3Embedding-3LayersDenseRNN42-Drop01-NoMeta-NoClassWeighted-WithEM-Train-L0.816583-NB5000.csv',\n",
       " '3Embedding-ESIM-Drop01-NoMeta-NoClassWeighted-NoEM-Train-L0.833565-NB5000.csv',\n",
       " 'WordSGNS-DenseCNN5Layers-NoMeta-3P-NoEM-NoClassWeighted-3Layers-Train-L0.838202-NB100000.csv',\n",
       " 'WordSGNS-ESIM-NoMeta-3P-NoEM-NoClassWeighted-3Layers-Train-L1.104962-NB100000.csv',\n",
       " 'WordTC-DenseCNN5Layers-NoMeta-3P-NoEM-NoClassWeighted-3Layers-Train-L0.8440-NB100000.csv',\n",
       " 'WordTC-DenseRNN-NoMeta-3P-NoEM-NoClassWeighted-3Layers-Train-L0.854586-NB100000.csv',\n",
       " 'WordTC-ESIM-NoMeta-3P-NoEM-NoClassWeighted-3Layers-Train-L0.374334-NB100000.csv',\n",
       " 'WordTC-Gated4GWindows-NoMeta-3P-NoEM-NoClassWeighted-3Layers-withEM-Train-L0.836860-NB100000.csv']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "oof_file_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "unrelated = pd.DataFrame()\n",
    "agreeds = pd.DataFrame()\n",
    "disagreeds = pd.DataFrame()\n",
    "\n",
    "#check_oofs = True\n",
    "check_oofs = False\n",
    "\n",
    "\n",
    "if check_oofs:\n",
    "    for i, oof in enumerate(oofs):\n",
    "        agreeds['oofs_agreed_{}'.format(i)] = oofs[i]['agreed'].values\n",
    "        unrelated['oofs_unrelated_{}'.format(i)] = oofs[i]['unrelated'].values\n",
    "        disagreeds['oofs_disagreeds_{}'.format(i)] = oofs[i]['disagreed'].values\n",
    "else:\n",
    "    for i, oof in enumerate(oofs):\n",
    "        agreeds['oofs_agreed_{}'.format(i)] = preds[i]['agreed'].values\n",
    "        unrelated['oofs_unrelated_{}'.format(i)] = preds[i]['unrelated'].values\n",
    "        disagreeds['oofs_disagreeds_{}'.format(i)] = preds[i]['disagreed'].values  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>oofs_agreed_0</th>\n",
       "      <th>oofs_agreed_1</th>\n",
       "      <th>oofs_agreed_2</th>\n",
       "      <th>oofs_agreed_3</th>\n",
       "      <th>oofs_agreed_4</th>\n",
       "      <th>oofs_agreed_5</th>\n",
       "      <th>oofs_agreed_6</th>\n",
       "      <th>oofs_agreed_7</th>\n",
       "      <th>oofs_agreed_8</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>oofs_agreed_0</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.982310</td>\n",
       "      <td>0.976632</td>\n",
       "      <td>0.959797</td>\n",
       "      <td>0.956807</td>\n",
       "      <td>0.961449</td>\n",
       "      <td>0.961597</td>\n",
       "      <td>0.959404</td>\n",
       "      <td>0.962629</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_agreed_1</th>\n",
       "      <td>0.982310</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.982479</td>\n",
       "      <td>0.958966</td>\n",
       "      <td>0.957473</td>\n",
       "      <td>0.961494</td>\n",
       "      <td>0.961780</td>\n",
       "      <td>0.961639</td>\n",
       "      <td>0.961288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_agreed_2</th>\n",
       "      <td>0.976632</td>\n",
       "      <td>0.982479</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.955476</td>\n",
       "      <td>0.959167</td>\n",
       "      <td>0.957512</td>\n",
       "      <td>0.961050</td>\n",
       "      <td>0.963775</td>\n",
       "      <td>0.956841</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_agreed_3</th>\n",
       "      <td>0.959797</td>\n",
       "      <td>0.958966</td>\n",
       "      <td>0.955476</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.982100</td>\n",
       "      <td>0.977313</td>\n",
       "      <td>0.970090</td>\n",
       "      <td>0.972992</td>\n",
       "      <td>0.973543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_agreed_4</th>\n",
       "      <td>0.956807</td>\n",
       "      <td>0.957473</td>\n",
       "      <td>0.959167</td>\n",
       "      <td>0.982100</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.971693</td>\n",
       "      <td>0.972289</td>\n",
       "      <td>0.979278</td>\n",
       "      <td>0.968705</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_agreed_5</th>\n",
       "      <td>0.961449</td>\n",
       "      <td>0.961494</td>\n",
       "      <td>0.957512</td>\n",
       "      <td>0.977313</td>\n",
       "      <td>0.971693</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.981701</td>\n",
       "      <td>0.981716</td>\n",
       "      <td>0.982047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_agreed_6</th>\n",
       "      <td>0.961597</td>\n",
       "      <td>0.961780</td>\n",
       "      <td>0.961050</td>\n",
       "      <td>0.970090</td>\n",
       "      <td>0.972289</td>\n",
       "      <td>0.981701</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.983917</td>\n",
       "      <td>0.982819</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_agreed_7</th>\n",
       "      <td>0.959404</td>\n",
       "      <td>0.961639</td>\n",
       "      <td>0.963775</td>\n",
       "      <td>0.972992</td>\n",
       "      <td>0.979278</td>\n",
       "      <td>0.981716</td>\n",
       "      <td>0.983917</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.978351</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_agreed_8</th>\n",
       "      <td>0.962629</td>\n",
       "      <td>0.961288</td>\n",
       "      <td>0.956841</td>\n",
       "      <td>0.973543</td>\n",
       "      <td>0.968705</td>\n",
       "      <td>0.982047</td>\n",
       "      <td>0.982819</td>\n",
       "      <td>0.978351</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               oofs_agreed_0  oofs_agreed_1  oofs_agreed_2  oofs_agreed_3  \\\n",
       "oofs_agreed_0       1.000000       0.982310       0.976632       0.959797   \n",
       "oofs_agreed_1       0.982310       1.000000       0.982479       0.958966   \n",
       "oofs_agreed_2       0.976632       0.982479       1.000000       0.955476   \n",
       "oofs_agreed_3       0.959797       0.958966       0.955476       1.000000   \n",
       "oofs_agreed_4       0.956807       0.957473       0.959167       0.982100   \n",
       "oofs_agreed_5       0.961449       0.961494       0.957512       0.977313   \n",
       "oofs_agreed_6       0.961597       0.961780       0.961050       0.970090   \n",
       "oofs_agreed_7       0.959404       0.961639       0.963775       0.972992   \n",
       "oofs_agreed_8       0.962629       0.961288       0.956841       0.973543   \n",
       "\n",
       "               oofs_agreed_4  oofs_agreed_5  oofs_agreed_6  oofs_agreed_7  \\\n",
       "oofs_agreed_0       0.956807       0.961449       0.961597       0.959404   \n",
       "oofs_agreed_1       0.957473       0.961494       0.961780       0.961639   \n",
       "oofs_agreed_2       0.959167       0.957512       0.961050       0.963775   \n",
       "oofs_agreed_3       0.982100       0.977313       0.970090       0.972992   \n",
       "oofs_agreed_4       1.000000       0.971693       0.972289       0.979278   \n",
       "oofs_agreed_5       0.971693       1.000000       0.981701       0.981716   \n",
       "oofs_agreed_6       0.972289       0.981701       1.000000       0.983917   \n",
       "oofs_agreed_7       0.979278       0.981716       0.983917       1.000000   \n",
       "oofs_agreed_8       0.968705       0.982047       0.982819       0.978351   \n",
       "\n",
       "               oofs_agreed_8  \n",
       "oofs_agreed_0       0.962629  \n",
       "oofs_agreed_1       0.961288  \n",
       "oofs_agreed_2       0.956841  \n",
       "oofs_agreed_3       0.973543  \n",
       "oofs_agreed_4       0.968705  \n",
       "oofs_agreed_5       0.982047  \n",
       "oofs_agreed_6       0.982819  \n",
       "oofs_agreed_7       0.978351  \n",
       "oofs_agreed_8       1.000000  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agreeds.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>oofs_disagreeds_0</th>\n",
       "      <th>oofs_disagreeds_1</th>\n",
       "      <th>oofs_disagreeds_2</th>\n",
       "      <th>oofs_disagreeds_3</th>\n",
       "      <th>oofs_disagreeds_4</th>\n",
       "      <th>oofs_disagreeds_5</th>\n",
       "      <th>oofs_disagreeds_6</th>\n",
       "      <th>oofs_disagreeds_7</th>\n",
       "      <th>oofs_disagreeds_8</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>oofs_disagreeds_0</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.954897</td>\n",
       "      <td>0.951416</td>\n",
       "      <td>0.912412</td>\n",
       "      <td>0.913741</td>\n",
       "      <td>0.912573</td>\n",
       "      <td>0.916230</td>\n",
       "      <td>0.912694</td>\n",
       "      <td>0.908535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_disagreeds_1</th>\n",
       "      <td>0.954897</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.964977</td>\n",
       "      <td>0.920113</td>\n",
       "      <td>0.918646</td>\n",
       "      <td>0.920680</td>\n",
       "      <td>0.921862</td>\n",
       "      <td>0.920311</td>\n",
       "      <td>0.911443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_disagreeds_2</th>\n",
       "      <td>0.951416</td>\n",
       "      <td>0.964977</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.919030</td>\n",
       "      <td>0.925045</td>\n",
       "      <td>0.915579</td>\n",
       "      <td>0.923305</td>\n",
       "      <td>0.926555</td>\n",
       "      <td>0.913171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_disagreeds_3</th>\n",
       "      <td>0.912412</td>\n",
       "      <td>0.920113</td>\n",
       "      <td>0.919030</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.967334</td>\n",
       "      <td>0.956456</td>\n",
       "      <td>0.945988</td>\n",
       "      <td>0.954167</td>\n",
       "      <td>0.943892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_disagreeds_4</th>\n",
       "      <td>0.913741</td>\n",
       "      <td>0.918646</td>\n",
       "      <td>0.925045</td>\n",
       "      <td>0.967334</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.946770</td>\n",
       "      <td>0.944932</td>\n",
       "      <td>0.961943</td>\n",
       "      <td>0.939010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_disagreeds_5</th>\n",
       "      <td>0.912573</td>\n",
       "      <td>0.920680</td>\n",
       "      <td>0.915579</td>\n",
       "      <td>0.956456</td>\n",
       "      <td>0.946770</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.965450</td>\n",
       "      <td>0.960278</td>\n",
       "      <td>0.955281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_disagreeds_6</th>\n",
       "      <td>0.916230</td>\n",
       "      <td>0.921862</td>\n",
       "      <td>0.923305</td>\n",
       "      <td>0.945988</td>\n",
       "      <td>0.944932</td>\n",
       "      <td>0.965450</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.963620</td>\n",
       "      <td>0.958197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_disagreeds_7</th>\n",
       "      <td>0.912694</td>\n",
       "      <td>0.920311</td>\n",
       "      <td>0.926555</td>\n",
       "      <td>0.954167</td>\n",
       "      <td>0.961943</td>\n",
       "      <td>0.960278</td>\n",
       "      <td>0.963620</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.954127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_disagreeds_8</th>\n",
       "      <td>0.908535</td>\n",
       "      <td>0.911443</td>\n",
       "      <td>0.913171</td>\n",
       "      <td>0.943892</td>\n",
       "      <td>0.939010</td>\n",
       "      <td>0.955281</td>\n",
       "      <td>0.958197</td>\n",
       "      <td>0.954127</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   oofs_disagreeds_0  oofs_disagreeds_1  oofs_disagreeds_2  \\\n",
       "oofs_disagreeds_0           1.000000           0.954897           0.951416   \n",
       "oofs_disagreeds_1           0.954897           1.000000           0.964977   \n",
       "oofs_disagreeds_2           0.951416           0.964977           1.000000   \n",
       "oofs_disagreeds_3           0.912412           0.920113           0.919030   \n",
       "oofs_disagreeds_4           0.913741           0.918646           0.925045   \n",
       "oofs_disagreeds_5           0.912573           0.920680           0.915579   \n",
       "oofs_disagreeds_6           0.916230           0.921862           0.923305   \n",
       "oofs_disagreeds_7           0.912694           0.920311           0.926555   \n",
       "oofs_disagreeds_8           0.908535           0.911443           0.913171   \n",
       "\n",
       "                   oofs_disagreeds_3  oofs_disagreeds_4  oofs_disagreeds_5  \\\n",
       "oofs_disagreeds_0           0.912412           0.913741           0.912573   \n",
       "oofs_disagreeds_1           0.920113           0.918646           0.920680   \n",
       "oofs_disagreeds_2           0.919030           0.925045           0.915579   \n",
       "oofs_disagreeds_3           1.000000           0.967334           0.956456   \n",
       "oofs_disagreeds_4           0.967334           1.000000           0.946770   \n",
       "oofs_disagreeds_5           0.956456           0.946770           1.000000   \n",
       "oofs_disagreeds_6           0.945988           0.944932           0.965450   \n",
       "oofs_disagreeds_7           0.954167           0.961943           0.960278   \n",
       "oofs_disagreeds_8           0.943892           0.939010           0.955281   \n",
       "\n",
       "                   oofs_disagreeds_6  oofs_disagreeds_7  oofs_disagreeds_8  \n",
       "oofs_disagreeds_0           0.916230           0.912694           0.908535  \n",
       "oofs_disagreeds_1           0.921862           0.920311           0.911443  \n",
       "oofs_disagreeds_2           0.923305           0.926555           0.913171  \n",
       "oofs_disagreeds_3           0.945988           0.954167           0.943892  \n",
       "oofs_disagreeds_4           0.944932           0.961943           0.939010  \n",
       "oofs_disagreeds_5           0.965450           0.960278           0.955281  \n",
       "oofs_disagreeds_6           1.000000           0.963620           0.958197  \n",
       "oofs_disagreeds_7           0.963620           1.000000           0.954127  \n",
       "oofs_disagreeds_8           0.958197           0.954127           1.000000  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "disagreeds.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>oofs_unrelated_0</th>\n",
       "      <th>oofs_unrelated_1</th>\n",
       "      <th>oofs_unrelated_2</th>\n",
       "      <th>oofs_unrelated_3</th>\n",
       "      <th>oofs_unrelated_4</th>\n",
       "      <th>oofs_unrelated_5</th>\n",
       "      <th>oofs_unrelated_6</th>\n",
       "      <th>oofs_unrelated_7</th>\n",
       "      <th>oofs_unrelated_8</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>oofs_unrelated_0</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.978855</td>\n",
       "      <td>0.972572</td>\n",
       "      <td>0.952930</td>\n",
       "      <td>0.949506</td>\n",
       "      <td>0.954771</td>\n",
       "      <td>0.955016</td>\n",
       "      <td>0.952586</td>\n",
       "      <td>0.955786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_unrelated_1</th>\n",
       "      <td>0.978855</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.979417</td>\n",
       "      <td>0.952496</td>\n",
       "      <td>0.950368</td>\n",
       "      <td>0.955304</td>\n",
       "      <td>0.955510</td>\n",
       "      <td>0.955456</td>\n",
       "      <td>0.954800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_unrelated_2</th>\n",
       "      <td>0.972572</td>\n",
       "      <td>0.979417</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.948158</td>\n",
       "      <td>0.952290</td>\n",
       "      <td>0.950251</td>\n",
       "      <td>0.954365</td>\n",
       "      <td>0.957542</td>\n",
       "      <td>0.949502</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_unrelated_3</th>\n",
       "      <td>0.952930</td>\n",
       "      <td>0.952496</td>\n",
       "      <td>0.948158</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.979002</td>\n",
       "      <td>0.973876</td>\n",
       "      <td>0.965568</td>\n",
       "      <td>0.969264</td>\n",
       "      <td>0.969451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_unrelated_4</th>\n",
       "      <td>0.949506</td>\n",
       "      <td>0.950368</td>\n",
       "      <td>0.952290</td>\n",
       "      <td>0.979002</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.967039</td>\n",
       "      <td>0.967506</td>\n",
       "      <td>0.975886</td>\n",
       "      <td>0.963249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_unrelated_5</th>\n",
       "      <td>0.954771</td>\n",
       "      <td>0.955304</td>\n",
       "      <td>0.950251</td>\n",
       "      <td>0.973876</td>\n",
       "      <td>0.967039</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.978773</td>\n",
       "      <td>0.978637</td>\n",
       "      <td>0.978786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_unrelated_6</th>\n",
       "      <td>0.955016</td>\n",
       "      <td>0.955510</td>\n",
       "      <td>0.954365</td>\n",
       "      <td>0.965568</td>\n",
       "      <td>0.967506</td>\n",
       "      <td>0.978773</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.981089</td>\n",
       "      <td>0.979624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_unrelated_7</th>\n",
       "      <td>0.952586</td>\n",
       "      <td>0.955456</td>\n",
       "      <td>0.957542</td>\n",
       "      <td>0.969264</td>\n",
       "      <td>0.975886</td>\n",
       "      <td>0.978637</td>\n",
       "      <td>0.981089</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.974969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>oofs_unrelated_8</th>\n",
       "      <td>0.955786</td>\n",
       "      <td>0.954800</td>\n",
       "      <td>0.949502</td>\n",
       "      <td>0.969451</td>\n",
       "      <td>0.963249</td>\n",
       "      <td>0.978786</td>\n",
       "      <td>0.979624</td>\n",
       "      <td>0.974969</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  oofs_unrelated_0  oofs_unrelated_1  oofs_unrelated_2  \\\n",
       "oofs_unrelated_0          1.000000          0.978855          0.972572   \n",
       "oofs_unrelated_1          0.978855          1.000000          0.979417   \n",
       "oofs_unrelated_2          0.972572          0.979417          1.000000   \n",
       "oofs_unrelated_3          0.952930          0.952496          0.948158   \n",
       "oofs_unrelated_4          0.949506          0.950368          0.952290   \n",
       "oofs_unrelated_5          0.954771          0.955304          0.950251   \n",
       "oofs_unrelated_6          0.955016          0.955510          0.954365   \n",
       "oofs_unrelated_7          0.952586          0.955456          0.957542   \n",
       "oofs_unrelated_8          0.955786          0.954800          0.949502   \n",
       "\n",
       "                  oofs_unrelated_3  oofs_unrelated_4  oofs_unrelated_5  \\\n",
       "oofs_unrelated_0          0.952930          0.949506          0.954771   \n",
       "oofs_unrelated_1          0.952496          0.950368          0.955304   \n",
       "oofs_unrelated_2          0.948158          0.952290          0.950251   \n",
       "oofs_unrelated_3          1.000000          0.979002          0.973876   \n",
       "oofs_unrelated_4          0.979002          1.000000          0.967039   \n",
       "oofs_unrelated_5          0.973876          0.967039          1.000000   \n",
       "oofs_unrelated_6          0.965568          0.967506          0.978773   \n",
       "oofs_unrelated_7          0.969264          0.975886          0.978637   \n",
       "oofs_unrelated_8          0.969451          0.963249          0.978786   \n",
       "\n",
       "                  oofs_unrelated_6  oofs_unrelated_7  oofs_unrelated_8  \n",
       "oofs_unrelated_0          0.955016          0.952586          0.955786  \n",
       "oofs_unrelated_1          0.955510          0.955456          0.954800  \n",
       "oofs_unrelated_2          0.954365          0.957542          0.949502  \n",
       "oofs_unrelated_3          0.965568          0.969264          0.969451  \n",
       "oofs_unrelated_4          0.967506          0.975886          0.963249  \n",
       "oofs_unrelated_5          0.978773          0.978637          0.978786  \n",
       "oofs_unrelated_6          1.000000          0.981089          0.979624  \n",
       "oofs_unrelated_7          0.981089          1.000000          0.974969  \n",
       "oofs_unrelated_8          0.979624          0.974969          1.000000  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unrelated.corr()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare Different Inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Only use oofs\n",
    "ensemble_trains = trains.values\n",
    "ensemble_tests = tests.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Use oof and meta features\n",
    "#ensemble_trains = np.concatenate((trains.values, trains_meta), axis=1)\n",
    "#ensemble_tests = np.concatenate((tests.values, tests_meta), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#ensemble_trains = trick_trains_features\n",
    "#ensemble_tests = trick_tests_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# char level\n",
    "ensemble_trains_char = trains.values[:, -9:]\n",
    "ensemble_tests_char = tests.values[:, -9:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# word level\n",
    "ensemble_trains_words = trains.values[:, :-9]\n",
    "ensemble_tests_words = tests.values[:, :-9]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RidgeRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import RidgeClassifier\n",
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_prob(v):\n",
    "    res = []\n",
    "    for d in v:\n",
    "        d = np.exp(d) / np.sum(np.exp(d))\n",
    "        res.append(d)\n",
    "    return np.array(res)\n",
    "\n",
    "def fit_every_feature_model(feature_data, label, feature_test_data, fold_count=3, predict=True):\n",
    "    predictions = np.zeros(shape=[len(feature_test_data), 3])\n",
    "    fold_size = len(feature_data) // fold_count\n",
    "    oofs = []\n",
    "    \n",
    "    log_loss = 0\n",
    "    for fold_id in range(fold_count):\n",
    "        print(\"Fold : \", fold_id)\n",
    "        fold_start = fold_size * fold_id\n",
    "        fold_end = fold_start + fold_size\n",
    "        if fold_id == fold_count - 1:\n",
    "            fold_end = len(feature_data)\n",
    "                \n",
    "        train_x = np.concatenate([feature_data[:fold_start], feature_data[fold_end:]])\n",
    "        train_y = np.concatenate([label[:fold_start], label[fold_end:]])\n",
    "\n",
    "        val_x = feature_data[fold_start:fold_end]\n",
    "        val_y = label[fold_start:fold_end]        \n",
    "        \n",
    "        clf = RidgeClassifier().fit(train_x, train_y)\n",
    "        print(\"Score\", clf.score(val_x, val_y))\n",
    "        \n",
    "        if predict:\n",
    "            prediction = get_prob(clf.decision_function(feature_test_data))\n",
    "            oof_prediction = get_prob(clf.decision_function(val_x))\n",
    "\n",
    "            score = metrics.log_loss(val_y, oof_prediction)\n",
    "            print(\"Fold\", fold_id, \"log loss\", score)\n",
    "            log_loss += score\n",
    "            oofs.append(oof_prediction)\n",
    "            predictions += prediction        \n",
    "        \n",
    "    predictions /= fold_count   \n",
    "    print(\"Training  Finish\")\n",
    "\n",
    "    return predictions, log_loss / fold_count, oofs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold :  0\n",
      "Score 0.883949461862424\n",
      "Fold 0 log loss 0.45107390737398706\n",
      "Fold :  1\n",
      "Score 0.8956793012010607\n",
      "Fold 1 log loss 0.4244238515981085\n",
      "Fold :  2\n",
      "Score 0.8860084230229294\n",
      "Fold 2 log loss 0.44675847126416923\n",
      "Fold :  3\n",
      "Score 0.8776165964748089\n",
      "Fold 3 log loss 0.4587890656624461\n",
      "Fold :  4\n",
      "Score 0.8936203400405553\n",
      "Fold 4 log loss 0.43617559931800554\n",
      "Fold :  5\n",
      "Score 0.8784900951489627\n",
      "Fold 5 log loss 0.45678059303021246\n",
      "Fold :  6\n",
      "Score 0.8672282015286227\n",
      "Fold 6 log loss 0.4682475966336388\n",
      "Fold :  7\n",
      "Score 0.8827328029948526\n",
      "Fold 7 log loss 0.4495128585172305\n",
      "Fold :  8\n",
      "Score 0.8858212447356106\n",
      "Fold 8 log loss 0.44782780430014735\n",
      "Fold :  9\n",
      "Score 0.8898212558879496\n",
      "Fold 9 log loss 0.4367892904853231\n",
      "Training  Finish\n"
     ]
    }
   ],
   "source": [
    "pred, log_loss, oofs = fit_every_feature_model(ensemble_trains, labels, ensemble_tests, fold_count=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def np_weighted_accuracy(y_true, y_pred):\n",
    "    weight = np.array([[1/16, 1/15, 1/5]])\n",
    "    norm = [(1/16) + (1/15) + (1/5)]\n",
    "    weight_mask = weight * y_true\n",
    "    weight_mask = np.max(weight_mask, axis=-1)\n",
    "    norms = np.sum(weight_mask)\n",
    "    \n",
    "    y_true = np.argmax(y_true, axis=-1)\n",
    "    y_pred = np.argmax(y_pred, axis=-1)\n",
    "    \n",
    "    res = ((y_true == y_pred) * weight_mask).sum() / norms\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "oofs = np.concatenate(oofs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "score = np_weighted_accuracy(to_categorical(labels), oofs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score 0.8599933011611254\n",
      "Predicting training results...\n",
      "Predicting labeled testing results...\n"
     ]
    }
   ],
   "source": [
    "print(\"score\", score)\n",
    "oofs_dir = \"../data/ensemble/oofs/\"\n",
    "output_dir = \"../data/ensemble/preds/\"\n",
    "onehot_pred_dir = \"../data/ensemble/nn_one_hot/\"\n",
    "\n",
    "model_submit_prefix = \"Ridge-Ensemble\"\n",
    "\n",
    "oofs_path = oofs_dir + model_submit_prefix\n",
    "output_path = output_dir + model_submit_prefix\n",
    "one_hot_pred_path = onehot_pred_dir + \"One-Hot\" + model_submit_prefix\n",
    "\n",
    "print(\"Predicting training results...\")\n",
    "oofs = pd.DataFrame({\"unrelated\": oofs[:, 0], \"agreed\": oofs[:, 1], \"disagreed\": oofs[:, 2]})\n",
    "submit_path = oofs_path + \"-Train-L{:4f}-NB{:d}.csv\".format(score, NB_WORDS)\n",
    "oofs.to_csv(submit_path, index=False)\n",
    "\n",
    "test_predicts = pd.DataFrame({\"unrelated\": pred[:, 0], \"agreed\": pred[:, 1], \"disagreed\": pred[:, 2]})\n",
    "submit_path = output_path + \"-L{:4f}-NB{:d}.csv\".format(score, NB_WORDS)\n",
    "test_predicts.to_csv(submit_path, index=False) # 0.3343\n",
    "\n",
    "print(\"Predicting labeled testing results...\")\n",
    "ids = pd.read_csv(\"../data/dataset/test.csv\")\n",
    "pred_labels = test_predicts.idxmax(axis=1)\n",
    "sub = pd.DataFrame({\"Id\": ids['id'].values, \"Category\": pred_labels})\n",
    "submit_path = one_hot_pred_path + \"-L{:4f}-NB{:d}.csv\".format(score, NB_WORDS)\n",
    "sub.to_csv(submit_path, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Logistic Ensembler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression, BayesianRidge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_prob(v):\n",
    "    res = []\n",
    "    for d in v:\n",
    "        d = np.exp(d) / np.sum(np.exp(d))\n",
    "        res.append(d)\n",
    "    return np.array(res)\n",
    "\n",
    "def fit_every_feature_model(feature_data, label, feature_test_data, fold_count=3, predict=True):\n",
    "    predictions = np.zeros(shape=[len(feature_test_data), 3])\n",
    "    fold_size = len(feature_data) // fold_count\n",
    "    oofs = []\n",
    "    \n",
    "    log_loss = 0\n",
    "    for fold_id in range(fold_count):\n",
    "        print(\"Fold : \", fold_id)\n",
    "        fold_start = fold_size * fold_id\n",
    "        fold_end = fold_start + fold_size\n",
    "        if fold_id == fold_count - 1:\n",
    "            fold_end = len(feature_data)\n",
    "                \n",
    "        train_x = np.concatenate([feature_data[:fold_start], feature_data[fold_end:]])\n",
    "        train_y = np.concatenate([label[:fold_start], label[fold_end:]])\n",
    "\n",
    "        val_x = feature_data[fold_start:fold_end]\n",
    "        val_y = label[fold_start:fold_end]        \n",
    "        \n",
    "        clf = LogisticRegression().fit(train_x, train_y)\n",
    "        print(\"Score\", clf.score(val_x, val_y))\n",
    "        \n",
    "        if predict:\n",
    "            prediction = clf.predict_proba(feature_test_data)\n",
    "            oof_prediction = clf.predict_proba(val_x)\n",
    "\n",
    "            score = metrics.log_loss(val_y, oof_prediction)\n",
    "            print(\"Fold\", fold_id, \"log loss\", score)\n",
    "            log_loss += score\n",
    "            oofs.append(oof_prediction)\n",
    "            predictions += prediction        \n",
    "        \n",
    "    predictions /= fold_count   \n",
    "    print(\"Training  Finish\")\n",
    "\n",
    "    return predictions, log_loss / fold_count, oofs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold :  0\n",
      "Score 0.8835439089065669\n",
      "Fold 0 log loss 0.2819937865780086\n",
      "Fold :  1\n",
      "Score 0.8953673373888629\n",
      "Fold 1 log loss 0.2561112554033099\n",
      "Fold :  2\n",
      "Score 0.8857276555919513\n",
      "Fold 2 log loss 0.2776414775934446\n",
      "Fold :  3\n",
      "Score 0.8782093277179848\n",
      "Fold 3 log loss 0.2920157093906666\n",
      "Fold :  4\n",
      "Score 0.8939323038527531\n",
      "Fold 4 log loss 0.2623087698842683\n",
      "Fold :  5\n",
      "Score 0.8786148806738419\n",
      "Fold 5 log loss 0.2838329860006761\n",
      "Fold :  6\n",
      "Score 0.8679769146778974\n",
      "Fold 6 log loss 0.3140044762949762\n",
      "Fold :  7\n",
      "Score 0.8819528934643581\n",
      "Fold 7 log loss 0.28211722834254943\n",
      "Fold :  8\n",
      "Score 0.8860084230229294\n",
      "Fold 8 log loss 0.2771825255620224\n",
      "Fold :  9\n",
      "Score 0.8891037838849549\n",
      "Fold 9 log loss 0.27080884958812057\n",
      "Training  Finish\n"
     ]
    }
   ],
   "source": [
    "pred, log_loss, oofs = fit_every_feature_model(ensemble_trains, labels, ensemble_tests, fold_count=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "oofs = np.concatenate(oofs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "score = np_weighted_accuracy(to_categorical(labels), oofs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8601914058883677"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score 0.8601914058883677\n",
      "Predicting training results...\n",
      "Predicting labeled testing results...\n"
     ]
    }
   ],
   "source": [
    "print(\"score\", score)\n",
    "oofs_dir = \"../data/ensemble/oofs/\"\n",
    "output_dir = \"../data/ensemble/preds/\"\n",
    "onehot_pred_dir = \"../data/ensemble/nn_one_hot/\"\n",
    "\n",
    "model_submit_prefix = \"Logistic-Ensemble\"\n",
    "\n",
    "oofs_path = oofs_dir + model_submit_prefix\n",
    "output_path = output_dir + model_submit_prefix\n",
    "one_hot_pred_path = onehot_pred_dir + \"One-Hot\" + model_submit_prefix\n",
    "\n",
    "print(\"Predicting training results...\")\n",
    "oofs = pd.DataFrame({\"unrelated\": oofs[:, 0], \"agreed\": oofs[:, 1], \"disagreed\": oofs[:, 2]})\n",
    "submit_path = oofs_path + \"-Train-L{:4f}-NB{:d}.csv\".format(score, NB_WORDS)\n",
    "oofs.to_csv(submit_path, index=False)\n",
    "\n",
    "test_predicts = pd.DataFrame({\"unrelated\": pred[:, 0], \"agreed\": pred[:, 1], \"disagreed\": pred[:, 2]})\n",
    "submit_path = output_path + \"-L{:4f}-NB{:d}.csv\".format(score, NB_WORDS)\n",
    "test_predicts.to_csv(submit_path, index=False) # 0.3343\n",
    "\n",
    "print(\"Predicting labeled testing results...\")\n",
    "ids = pd.read_csv(\"../data/dataset/test.csv\")\n",
    "pred_labels = test_predicts.idxmax(axis=1)\n",
    "sub = pd.DataFrame({\"Id\": ids['id'].values, \"Category\": pred_labels})\n",
    "submit_path = one_hot_pred_path + \"-L{:4f}-NB{:d}.csv\".format(score, NB_WORDS)\n",
    "sub.to_csv(submit_path, index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 1,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
